/*
  Copyright (C) 2011, Kyungjoo Kim
  
  This file is part of LINAL (LINear ALgebra)
  
  All rights reserved.

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.
  
  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.
  
  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  Also add information on how to contact you by electronic and paper mail.
  
  
  Kyungjoo Kim
  iamkyungjoo@gmail.com
*/
#include "gpu_perform.hxx"

int main(int argc, char **argv) {

  if (argc != 10) {
    printf("Try :: lu_nopiv [n_thread] [n_thread_gpu] [n_gpu] [balance] [gpu_mn] [cpu_mn] [n_dof] [n_itr] [n_block]\n");
    return 0;
  }

  // ---------------------------------------
  // ** Initialization   
  int
    n_thread     = atoi( (argv[1]) ),
    n_thread_gpu = atoi( (argv[2]) ),
    n_gpu        = atoi( (argv[3]) ),
    n_balance    = atoi( (argv[4]) ),
    gpu_mn    = atoi( (argv[5]) ),
    cpu_mn       = atoi( (argv[6]) ),
    n_dof        = atoi( (argv[7]) ),
    n_itr        = atoi( (argv[8]) ),
    n_block      = atoi( (argv[9]) );

  int flame = 0, linal = 1, n_device;
  cudaGetDeviceCount( &n_device );

  printf("** TEST ENVIRONMENT **\n");
  printf("NDOF      = %d\n", n_dof);
  printf("Gpu_Mn = %d\n", gpu_mn);
  printf("N thread  = %d\n", n_thread);
  printf("N device  = %d\n", n_device);
  printf("N gpu     = %d\n", n_gpu);
  printf("CPU block = %d\n", cpu_mn);
  printf("Iteration = %d\n", n_itr);
  printf("Balance   = %d\n", n_balance);

  int b_mn[2];
  b_mn[0] = b_mn[1] = gpu_mn;

  FLA_Init();

  // ---------------------------------------
  // ** Environement setting   
  double flop = linal::get_flop_lu( 0, n_dof, n_dof );

  // ---------------------------------------
  // ** Matrices   
  double
    t_base, t_flash_repack, t_linal_repack, t_temp,
    t_flash_decompose, t_linal_decompose;

  linal::Flat_ A, p;
  A.create(TEST_DATATYPE, n_dof, n_dof);
  p.create(LINAL_INT, n_dof, 1);
  FLA_Random_matrix( ~A );

  FLA_Obj hA_fla, hp_fla;
  t_base = FLA_Clock();
  if (flame) {
    FLASH_Obj_create_hier_copy_of_flat(~A, 1, (dim_t*)b_mn, &hA_fla);
    FLASH_Obj_create_hier_copy_of_flat(~p, 1, (dim_t*)b_mn, &hp_fla);
  }
  t_flash_repack = FLA_Clock()-t_base;

  linal::Hier_ hA_linal, hp_linal;
  t_base = FLA_Clock();
  hA_linal.create(A, gpu_mn, gpu_mn);
  hp_linal.create(p, gpu_mn, gpu_mn);
  t_linal_repack = FLA_Clock()-t_base;

  // ---------------------------------------
  // ** FLASH
  if (flame) {
    FLASH_Queue_set_num_threads(n_gpu);
    FLASH_Queue_set_sorting(TRUE);
    FLASH_Queue_set_caching(TRUE);
    FLASH_Queue_set_data_affinity(FLASH_QUEUE_AFFINITY_NONE);
    FLASH_Queue_enable_gpu();
    FLASH_Queue_set_gpu_num_blocks(n_block);

    t_flash_decompose = MAX_TIME;

    for (int q=0;q<n_itr;++q) {
      printf("*** FLASH::LU PIV BEGIN ***\n");
      t_base = FLA_Clock();
      FLASH_Queue_begin();
      FLASH_LU_piv(hA_fla, hp_fla);
      FLASH_Queue_end();
      t_temp = FLA_Clock()-t_base;
      printf("*** FLASH::LU PIV END *** time %lf\n", t_temp);

      t_flash_decompose = min(t_temp, t_flash_decompose);
    }
    
    FLASH_Queue_disable_gpu();
  }

  // ---------------------------------------
  // ** LINAL
  if (linal) {
    omp_set_num_threads(n_thread);

    int n_item_bin = n_block, n_item_work = 1;
    linal::init_gpu(n_thread_gpu, n_gpu, n_item_bin, n_item_work);
    linal::set_computing_model( TEST_COMPUTING );
    linal::set_cpu_blocksize( cpu_mn );
    linal::set_work_balance(n_balance);
    linal::create_flat_gpu( TEST_DATATYPE, b_mn[0], b_mn[1] );

    t_linal_decompose = MAX_TIME;
    for (int q=0;q<n_itr;++q) {
      printf("*** LINAL::LU PIV BEGIN ***\n");
      t_base = FLA_Clock();
#pragma omp parallel 
      {
#pragma omp single nowait
	linal::dense::lu_piv(hA_linal, hp_linal);
      }
      t_temp = FLA_Clock()-t_base;
      printf("*** LINAL::LU PIV END ***\n");
      
      t_linal_decompose = min(t_temp, t_linal_decompose);
    }

    linal::disp_gpu_data_access(stdout);
    linal::disp_operation_stat(stdout);

    linal::set_computing_model( LINAL_CPU );
    linal::free_flat_gpu();
    linal::finalize_gpu();
    
    switch (TEST_COMPUTING) {
    case LINAL_CPU_GPU:  printf("CPU + GPU BLAS\n");    break;
    case LINAL_GPU:      printf("GPU BLAS\n");          break;
    default:	       printf("CPU ONLY\n");          break;
    }
  }

  printf("----------------------------------------------\n");
  printf("*** Report LU incpiv ***\n");
  printf("Ndof       = %d\n", n_dof);
  printf("BLK        = %d, %d\n", gpu_mn, cpu_mn);
  printf("Nthread    = %d\n", n_thread);
  printf("Ngpu       = %d\n", n_gpu);
  printf("Niteration = %d\n", n_itr);
  printf("----------------------------------------------\n");
  printf("Time Supermatrix   = %6.3lf [sec]\n", t_flash_decompose);
  printf("Time LINAL         = %6.3lf [sec]\n", t_linal_decompose);
  printf("----------------------------------------------\n");
  printf("FLOPS Supermatrix  = %6.3lf [Gflops]\n",
         flop/t_flash_decompose/1.0e9);
  printf("FLOPS LINAL        = %6.3lf [Gflops]\n",
         flop/t_linal_decompose/1.0e9);
  printf("----------------------------------------------\n");
  printf("With repacking cost \n");
  printf("FLOPS Supermatrix  = %6.3lf [Gflops]\n",
         flop/(t_flash_decompose+t_flash_repack)/1.0e9);
  printf("FLOPS LINAL        = %6.3lf [Gflops]\n",
         flop/(t_linal_decompose+t_linal_repack)/1.0e9);
  printf("----------------------------------------------\n");
  
  // ---------------------------------------
  // ** Matrices   
  if (flame) {
    FLASH_Obj_free(&hA_fla);
    FLASH_Obj_free(&hp_fla);
  }
  hA_linal.free();
  hp_linal.free();
  A.free();
  p.free();

  printf("*** TEST FINISHED ***\n");

  // ---------------------------------------
  // ** Finalization
  FLA_Finalize();
  return 0;
}
